{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking and describing the generated data\n",
    "\n",
    "It is always beneficial to add a notebook that quickly looks into the data to help you remember, which data you collected and if it actually looks correct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from algbench import describe, read_as_pandas, Benchmark\n",
    "from _conf import EXPERIMENT_DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An entry in the database can look like this:\n",
      "_____________________________________________\n",
      " result:\n",
      "| num_nodes: 100\n",
      "| lower_bound: 87630783.0\n",
      "| objective: 93096532.0\n",
      " timestamp: 2023-11-17T10:20:43.172028\n",
      " runtime: 90.37890768051147\n",
      " stdout: []\n",
      " stderr: []\n",
      " logging: [{'name': 'Evaluation', 'msg': 'Building model.', 'args': [], 'levelname': 'I...\n",
      " env_fingerprint: 7de002f2d293f6cb7c59ec1a6de2e660bb383ef9\n",
      " args_fingerprint: f68b86df4d4b794819939d1619123b57112504ee\n",
      " parameters:\n",
      "| func: run_solver\n",
      "| args:\n",
      "|| instance_name: random_euclidean_100_0\n",
      "|| time_limit: 90\n",
      "|| strategy: CpSatTspSolverV1\n",
      "|| opt_tol: 0.001\n",
      " argv: ['/ibr/home/krupke/anaconda3/envs/mo310/lib/python3.10/site-packages/slurmina...\n",
      " env:\n",
      "| hostname: algra02\n",
      "| python_version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]\n",
      "| python: /ibr/home/krupke/anaconda3/envs/mo310/bin/python3\n",
      "| cwd: /misc/ibr/home/krupke/cpsat-primer/examples/tsp_evaluation\n",
      "| git_revision: d56ffe5c790b54d038b0495f722e304dcbfa845e\n",
      "| python_file: /ibr/home/krupke/anaconda3/envs/mo310/lib/python3.10/site-packages/slurminade...\n",
      "______________________________________________\n",
      "Note that this is only based on the first entry, other entries could differ.\n"
     ]
    }
   ],
   "source": [
    "describe(EXPERIMENT_DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instance_name</th>\n",
       "      <th>num_nodes</th>\n",
       "      <th>time_limit</th>\n",
       "      <th>strategy</th>\n",
       "      <th>opt_tol</th>\n",
       "      <th>runtime</th>\n",
       "      <th>objective</th>\n",
       "      <th>lower_bound</th>\n",
       "      <th>opt_gap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>random_euclidean_25_0</td>\n",
       "      <td>25</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverV1</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.357365</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>9.635576e+07</td>\n",
       "      <td>0.000121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>random_euclidean_25_0</td>\n",
       "      <td>25</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverDantzig</td>\n",
       "      <td>0.001</td>\n",
       "      <td>1.572138</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <td>random_euclidean_25_0</td>\n",
       "      <td>25</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverMtz</td>\n",
       "      <td>0.001</td>\n",
       "      <td>9.163427</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1820</th>\n",
       "      <td>random_euclidean_25_0</td>\n",
       "      <td>25</td>\n",
       "      <td>90</td>\n",
       "      <td>GurobiTspSolver</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.585014</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>9.636744e+07</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1260</th>\n",
       "      <td>random_euclidean_25_1</td>\n",
       "      <td>25</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverV1</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.077645</td>\n",
       "      <td>9.077355e+07</td>\n",
       "      <td>9.077355e+07</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1885</th>\n",
       "      <td>random_euclidean_500_8</td>\n",
       "      <td>500</td>\n",
       "      <td>90</td>\n",
       "      <td>GurobiTspSolver</td>\n",
       "      <td>0.001</td>\n",
       "      <td>92.770625</td>\n",
       "      <td>inf</td>\n",
       "      <td>6.760793e+07</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>random_euclidean_500_9</td>\n",
       "      <td>500</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverV1</td>\n",
       "      <td>0.001</td>\n",
       "      <td>95.620484</td>\n",
       "      <td>1.430933e+10</td>\n",
       "      <td>6.014955e+07</td>\n",
       "      <td>236.895928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1025</th>\n",
       "      <td>random_euclidean_500_9</td>\n",
       "      <td>500</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverDantzig</td>\n",
       "      <td>0.001</td>\n",
       "      <td>115.235823</td>\n",
       "      <td>1.596116e+10</td>\n",
       "      <td>6.842889e+07</td>\n",
       "      <td>232.251712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1030</th>\n",
       "      <td>random_euclidean_500_9</td>\n",
       "      <td>500</td>\n",
       "      <td>90</td>\n",
       "      <td>CpSatTspSolverMtz</td>\n",
       "      <td>0.001</td>\n",
       "      <td>125.114764</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2330</th>\n",
       "      <td>random_euclidean_500_9</td>\n",
       "      <td>500</td>\n",
       "      <td>90</td>\n",
       "      <td>GurobiTspSolver</td>\n",
       "      <td>0.001</td>\n",
       "      <td>92.104872</td>\n",
       "      <td>inf</td>\n",
       "      <td>7.193185e+07</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>480 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               instance_name  num_nodes  time_limit               strategy  \\\n",
       "435    random_euclidean_25_0         25          90       CpSatTspSolverV1   \n",
       "440    random_euclidean_25_0         25          90  CpSatTspSolverDantzig   \n",
       "445    random_euclidean_25_0         25          90      CpSatTspSolverMtz   \n",
       "1820   random_euclidean_25_0         25          90        GurobiTspSolver   \n",
       "1260   random_euclidean_25_1         25          90       CpSatTspSolverV1   \n",
       "...                      ...        ...         ...                    ...   \n",
       "1885  random_euclidean_500_8        500          90        GurobiTspSolver   \n",
       "1020  random_euclidean_500_9        500          90       CpSatTspSolverV1   \n",
       "1025  random_euclidean_500_9        500          90  CpSatTspSolverDantzig   \n",
       "1030  random_euclidean_500_9        500          90      CpSatTspSolverMtz   \n",
       "2330  random_euclidean_500_9        500          90        GurobiTspSolver   \n",
       "\n",
       "      opt_tol     runtime     objective   lower_bound     opt_gap  \n",
       "435     0.001    0.357365  9.636744e+07  9.635576e+07    0.000121  \n",
       "440     0.001    1.572138  9.636744e+07  9.636744e+07    0.000000  \n",
       "445     0.001    9.163427  9.636744e+07  9.636744e+07    0.000000  \n",
       "1820    0.001    0.585014  9.636744e+07  9.636744e+07    0.000000  \n",
       "1260    0.001    0.077645  9.077355e+07  9.077355e+07    0.000000  \n",
       "...       ...         ...           ...           ...         ...  \n",
       "1885    0.001   92.770625           inf  6.760793e+07         inf  \n",
       "1020    0.001   95.620484  1.430933e+10  6.014955e+07  236.895928  \n",
       "1025    0.001  115.235823  1.596116e+10  6.842889e+07  232.251712  \n",
       "1030    0.001  125.114764           inf  0.000000e+00         inf  \n",
       "2330    0.001   92.104872           inf  7.193185e+07         inf  \n",
       "\n",
       "[480 rows x 9 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = read_as_pandas(\n",
    "    EXPERIMENT_DATA,\n",
    "    lambda entry: {\n",
    "        \"instance_name\": entry[\"parameters\"][\"args\"][\"instance_name\"],\n",
    "        \"num_nodes\": entry[\"result\"][\"num_nodes\"],\n",
    "        \"time_limit\": entry[\"parameters\"][\"args\"][\"time_limit\"],\n",
    "        \"strategy\": entry[\"parameters\"][\"args\"][\"strategy\"],\n",
    "        \"opt_tol\": entry[\"parameters\"][\"args\"][\"opt_tol\"],\n",
    "        \"runtime\": entry[\"runtime\"],\n",
    "        \"objective\": entry[\"result\"][\"objective\"],\n",
    "        \"lower_bound\": entry[\"result\"][\"lower_bound\"],\n",
    "    },\n",
    ")\n",
    "t.drop_duplicates(inplace=True, subset=[\"instance_name\", \"num_nodes\", \"strategy\"])\n",
    "t[\"opt_gap\"] = (t[\"objective\"] - t[\"lower_bound\"]) / t[\"lower_bound\"]\n",
    "t.sort_values([\"num_nodes\", \"instance_name\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=====================================\n",
      "Set parameter Username\n",
      "Academic license - for non-commercial use only - expires 2024-11-16\n",
      "Set parameter TimeLimit to value 90\n",
      "Set parameter LazyConstraints to value 1\n",
      "Set parameter MIPGap to value 0.001\n",
      "Gurobi Optimizer version 10.0.3 build v10.0.3rc0 (linux64)\n",
      "\n",
      "CPU model: AMD Ryzen 9 7900 12-Core Processor, instruction set [SSE2|AVX|AVX2|AVX512]\n",
      "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n",
      "\n",
      "Optimize a model with 100 rows, 4950 columns and 9900 nonzeros\n",
      "Model fingerprint: 0x6d5410e4\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [1e+00, 1e+00]\n",
      "  Objective range  [9e+03, 1e+08]\n",
      "  Bounds range     [1e+00, 1e+00]\n",
      "  RHS range        [2e+00, 2e+00]\n",
      "Presolve time: 0.00s\n",
      "Presolved: 100 rows, 4950 columns, 9900 nonzeros\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "\n",
      "Root relaxation: objective 6.600813e+07, 105 iterations, 0.00 seconds (0.00 work units)\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0 6.6008e+07    0   18          - 6.6008e+07      -     -    0s\n",
      "     0     0 7.1219e+07    0   14          - 7.1219e+07      -     -    0s\n",
      "     0     0 7.1461e+07    0   12          - 7.1461e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     2 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "*  111   119              10    7.575806e+07 7.2269e+07  4.61%  13.0    0s\n",
      "*  128   119              10    7.520331e+07 7.2269e+07  3.90%  11.8    0s\n",
      "H  136    89                    7.437426e+07 7.2540e+07  2.47%  11.2    0s\n",
      "\n",
      "Cutting planes:\n",
      "  Zero half: 11\n",
      "  Lazy constraints: 73\n",
      "\n",
      "Explored 207 nodes (1964 simplex iterations) in 0.87 seconds (0.12 work units)\n",
      "Thread count was 24 (of 24 available processors)\n",
      "\n",
      "Solution count 3: 7.43743e+07 7.52033e+07 7.57581e+07 \n",
      "\n",
      "Optimal solution found (tolerance 1.00e-03)\n",
      "Best objective 7.437426400000e+07, best bound 7.437426400000e+07, gap 0.0000%\n",
      "\n",
      "User-callback calls 609, time in user-callback 0.28 sec\n",
      "\n",
      "\n",
      "=====================================\n",
      "=====================================\n",
      "Set parameter TimeLimit to value 90\n",
      "Set parameter LazyConstraints to value 1\n",
      "Set parameter MIPGap to value 0.01\n",
      "Gurobi Optimizer version 10.0.3 build v10.0.3rc0 (linux64)\n",
      "\n",
      "CPU model: AMD Ryzen 9 7900 12-Core Processor, instruction set [SSE2|AVX|AVX2|AVX512]\n",
      "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n",
      "\n",
      "Optimize a model with 100 rows, 4950 columns and 9900 nonzeros\n",
      "Model fingerprint: 0x6d5410e4\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [1e+00, 1e+00]\n",
      "  Objective range  [9e+03, 1e+08]\n",
      "  Bounds range     [1e+00, 1e+00]\n",
      "  RHS range        [2e+00, 2e+00]\n",
      "Presolve time: 0.00s\n",
      "Presolved: 100 rows, 4950 columns, 9900 nonzeros\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "\n",
      "Root relaxation: objective 6.600813e+07, 105 iterations, 0.00 seconds (0.00 work units)\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0 6.6008e+07    0   18          - 6.6008e+07      -     -    0s\n",
      "     0     0 7.1219e+07    0   14          - 7.1219e+07      -     -    0s\n",
      "     0     0 7.1461e+07    0   12          - 7.1461e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     2 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "*  111   119              10    7.575806e+07 7.2269e+07  4.61%  13.0    0s\n",
      "*  128   119              10    7.520331e+07 7.2269e+07  3.90%  11.8    0s\n",
      "H  136    89                    7.437426e+07 7.2540e+07  2.47%  11.2    0s\n",
      "\n",
      "Cutting planes:\n",
      "  Zero half: 11\n",
      "  Lazy constraints: 73\n",
      "\n",
      "Explored 207 nodes (1964 simplex iterations) in 0.86 seconds (0.12 work units)\n",
      "Thread count was 24 (of 24 available processors)\n",
      "\n",
      "Solution count 3: 7.43743e+07 7.52033e+07 7.57581e+07 \n",
      "\n",
      "Optimal solution found (tolerance 1.00e-02)\n",
      "Best objective 7.437426400000e+07, best bound 7.437426400000e+07, gap 0.0000%\n",
      "\n",
      "User-callback calls 607, time in user-callback 0.23 sec\n",
      "\n",
      "\n",
      "=====================================\n",
      "=====================================\n",
      "Set parameter TimeLimit to value 90\n",
      "Set parameter LazyConstraints to value 1\n",
      "Set parameter MIPGap to value 0.05\n",
      "Gurobi Optimizer version 10.0.3 build v10.0.3rc0 (linux64)\n",
      "\n",
      "CPU model: AMD Ryzen 9 7900 12-Core Processor, instruction set [SSE2|AVX|AVX2|AVX512]\n",
      "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n",
      "\n",
      "Optimize a model with 100 rows, 4950 columns and 9900 nonzeros\n",
      "Model fingerprint: 0x6d5410e4\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [1e+00, 1e+00]\n",
      "  Objective range  [9e+03, 1e+08]\n",
      "  Bounds range     [1e+00, 1e+00]\n",
      "  RHS range        [2e+00, 2e+00]\n",
      "Presolve time: 0.00s\n",
      "Presolved: 100 rows, 4950 columns, 9900 nonzeros\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "\n",
      "Root relaxation: objective 6.600813e+07, 105 iterations, 0.00 seconds (0.00 work units)\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0 6.6008e+07    0   18          - 6.6008e+07      -     -    0s\n",
      "     0     0 7.1219e+07    0   14          - 7.1219e+07      -     -    0s\n",
      "     0     0 7.1461e+07    0   12          - 7.1461e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     2 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "*  111   119              10    7.575806e+07 7.2269e+07  4.61%  13.0    0s\n",
      "*  128   119              10    7.520331e+07 7.2269e+07  3.90%  11.8    0s\n",
      "\n",
      "Cutting planes:\n",
      "  Zero half: 11\n",
      "  Lazy constraints: 73\n",
      "\n",
      "Explored 135 nodes (1677 simplex iterations) in 0.81 seconds (0.11 work units)\n",
      "Thread count was 24 (of 24 available processors)\n",
      "\n",
      "Solution count 2: 7.52033e+07 7.57581e+07 \n",
      "\n",
      "Optimal solution found (tolerance 5.00e-02)\n",
      "Best objective 7.520331400000e+07, best bound 7.226916633333e+07, gap 3.9016%\n",
      "\n",
      "User-callback calls 460, time in user-callback 0.23 sec\n",
      "\n",
      "\n",
      "=====================================\n",
      "=====================================\n",
      "Set parameter TimeLimit to value 90\n",
      "Set parameter LazyConstraints to value 1\n",
      "Set parameter MIPGap to value 0.1\n",
      "Gurobi Optimizer version 10.0.3 build v10.0.3rc0 (linux64)\n",
      "\n",
      "CPU model: AMD Ryzen 9 7900 12-Core Processor, instruction set [SSE2|AVX|AVX2|AVX512]\n",
      "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n",
      "\n",
      "Optimize a model with 100 rows, 4950 columns and 9900 nonzeros\n",
      "Model fingerprint: 0x6d5410e4\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [1e+00, 1e+00]\n",
      "  Objective range  [9e+03, 1e+08]\n",
      "  Bounds range     [1e+00, 1e+00]\n",
      "  RHS range        [2e+00, 2e+00]\n",
      "Presolve time: 0.00s\n",
      "Presolved: 100 rows, 4950 columns, 9900 nonzeros\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "\n",
      "Root relaxation: objective 6.600813e+07, 105 iterations, 0.00 seconds (0.00 work units)\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0 6.6008e+07    0   18          - 6.6008e+07      -     -    0s\n",
      "     0     0 7.1219e+07    0   14          - 7.1219e+07      -     -    0s\n",
      "     0     0 7.1461e+07    0   12          - 7.1461e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     2 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "*  111   119              10    7.575806e+07 7.2269e+07  4.61%  13.0    0s\n",
      "*  128   119              10    7.520331e+07 7.2269e+07  3.90%  11.8    0s\n",
      "\n",
      "Cutting planes:\n",
      "  Zero half: 11\n",
      "  Lazy constraints: 73\n",
      "\n",
      "Explored 135 nodes (1677 simplex iterations) in 0.80 seconds (0.11 work units)\n",
      "Thread count was 24 (of 24 available processors)\n",
      "\n",
      "Solution count 2: 7.52033e+07 7.57581e+07 \n",
      "\n",
      "Optimal solution found (tolerance 1.00e-01)\n",
      "Best objective 7.520331400000e+07, best bound 7.226916633333e+07, gap 3.9016%\n",
      "\n",
      "User-callback calls 460, time in user-callback 0.30 sec\n",
      "\n",
      "\n",
      "=====================================\n",
      "=====================================\n",
      "Set parameter TimeLimit to value 90\n",
      "Set parameter LazyConstraints to value 1\n",
      "Set parameter MIPGap to value 0.25\n",
      "Gurobi Optimizer version 10.0.3 build v10.0.3rc0 (linux64)\n",
      "\n",
      "CPU model: AMD Ryzen 9 7900 12-Core Processor, instruction set [SSE2|AVX|AVX2|AVX512]\n",
      "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n",
      "\n",
      "Optimize a model with 100 rows, 4950 columns and 9900 nonzeros\n",
      "Model fingerprint: 0x6d5410e4\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [1e+00, 1e+00]\n",
      "  Objective range  [9e+03, 1e+08]\n",
      "  Bounds range     [1e+00, 1e+00]\n",
      "  RHS range        [2e+00, 2e+00]\n",
      "Presolve time: 0.00s\n",
      "Presolved: 100 rows, 4950 columns, 9900 nonzeros\n",
      "Variable types: 0 continuous, 4950 integer (4950 binary)\n",
      "\n",
      "Root relaxation: objective 6.600813e+07, 105 iterations, 0.00 seconds (0.00 work units)\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0 6.6008e+07    0   18          - 6.6008e+07      -     -    0s\n",
      "     0     0 7.1219e+07    0   14          - 7.1219e+07      -     -    0s\n",
      "     0     0 7.1461e+07    0   12          - 7.1461e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     0 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "     0     2 7.1572e+07    0   10          - 7.1572e+07      -     -    0s\n",
      "*  111   119              10    7.575806e+07 7.2269e+07  4.61%  13.0    0s\n",
      "*  128   119              10    7.520331e+07 7.2269e+07  3.90%  11.8    0s\n",
      "\n",
      "Cutting planes:\n",
      "  Zero half: 11\n",
      "  Lazy constraints: 73\n",
      "\n",
      "Explored 135 nodes (1677 simplex iterations) in 0.80 seconds (0.11 work units)\n",
      "Thread count was 24 (of 24 available processors)\n",
      "\n",
      "Solution count 2: 7.52033e+07 7.57581e+07 \n",
      "\n",
      "Optimal solution found (tolerance 2.50e-01)\n",
      "Best objective 7.520331400000e+07, best bound 7.226916633333e+07, gap 3.9016%\n",
      "\n",
      "User-callback calls 459, time in user-callback 0.29 sec\n",
      "\n",
      "\n",
      "=====================================\n"
     ]
    }
   ],
   "source": [
    "for entry in Benchmark(EXPERIMENT_DATA):\n",
    "    if (\n",
    "        entry[\"parameters\"][\"args\"][\"instance_name\"] == \"random_euclidean_100_1\"\n",
    "        and entry[\"parameters\"][\"args\"][\"strategy\"] == \"GurobiTspSolver\"\n",
    "    ):\n",
    "        print(\"=====================================\")\n",
    "        stdout = \"\".join(e[1] for e in entry[\"stdout\"])\n",
    "        stderr = \"\".join(e[1] for e in entry[\"stderr\"])\n",
    "        print(stdout)\n",
    "        print(stderr)\n",
    "        if not stdout.strip():\n",
    "            print(\"No stdout\")\n",
    "        print(\"=====================================\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check for errors in the data\n",
    "\n",
    "You always want to check if the results you got are actually feasible. Errors easily happen and are not always visible on the plots.\n",
    "Thus, you want to do some basic checks to detect errors early on. For example, you could accidentally have swapped lower and upper bounds in the data generation process.\n",
    "Depending on your plots, this may not be visible, and you may end up comparing the wrong data and draw the wrong conclusions.\n",
    "Or, you could have accidentally swapped runtime and objective values, which could look reasonable in the data as the runtime and the objective often increase with the instance size.\n",
    "\n",
    "A very basic check is to check if the best lower and upper bounds do not contradict each other. Many errors will be caught by this check. However, you often need some tolerance to account for numerical errors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (t.dropna()[\"opt_gap\"] >= -0.0001).all(), \"Optimality gap is negative!\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instance_name</th>\n",
       "      <th>num_nodes</th>\n",
       "      <th>time_limit</th>\n",
       "      <th>strategy</th>\n",
       "      <th>opt_tol</th>\n",
       "      <th>runtime</th>\n",
       "      <th>objective</th>\n",
       "      <th>lower_bound</th>\n",
       "      <th>opt_gap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [instance_name, num_nodes, time_limit, strategy, opt_tol, runtime, objective, lower_bound, opt_gap]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Always make sure that your results are not trivially wrong\n",
    "#  - e.g. lower bound is higher than objective\n",
    "max_lb = t.groupby([\"instance_name\"])[\"lower_bound\"].max()\n",
    "min_obj = t.groupby([\"instance_name\"])[\"objective\"].min()\n",
    "eps = 0.0001  # some tolerance is needed when working with floats.\n",
    "bad_instances = max_lb[max_lb - min_obj > eps * max_lb].index.to_list()\n",
    "from IPython.display import display\n",
    "\n",
    "display(t[t[\"instance_name\"].isin(bad_instances)])\n",
    "assert len(bad_instances) == 0, \"Bad instances detected: {}\".format(bad_instances)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mo310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
